# -*- coding: utf-8 -*-
# @Time    : 2021/9/22 17:26
# @Author  : ZSQ
# @Email   : zsq199170918@163.com
# @FileName: pipelines.py
# @Software: PyCharm

import redis
from HifoXf.utils.redis_pool import PROXY_REDIS_KEY, POOL
from HifoXf.utils.mongoclient import MongodbClient
from HifoXf.items import ProjectItem
import logging
from HifoXf.utils.redis_case_filter import get_filter_flag
import datetime

logger = logging.getLogger(__name__)


class RedisConnPipeline(object):
    """
    redis连接
    """

    def open_spider(self, spider):
        spider.redis_conn = redis.StrictRedis(connection_pool=POOL)
        spider.redis_conn.delete(PROXY_REDIS_KEY)

    def close_spider(self, spider):
        spider.redis_conn.close()


class MongoClientPipeline(object):
    """
    mongo入库
    """

    def open_spider(self, spider):
        self.clo = MongodbClient()

    def close_spider(self, spider):
        self.clo.close()

    def process_item(self, item, spider):
        # 贝壳
        if spider.name == 'beike':
            self.clo.set_db('newhours')
            if isinstance(item, ProjectItem):
                self.clo.set_col('beike_xf')
                self.clo.update_one({'projectId': item['projectId'], 'preSalePermit': item['preSalePermit']},
                                    dict(item))
        # 安居客
        if spider.name == 'anjuke':
            self.clo.set_db('newhours')
            if isinstance(item, ProjectItem):
                self.clo.set_col('anjuke_xf')
                self.clo.update_one({'projectId': item['projectId'], 'preSalePermit': item['preSalePermit']},
                                    dict(item))


class BeikePipeline(object):
    """
    贝壳新房
    """

    def open_spider(self, spider):
        if spider.name == 'beike':
            case_id_filter = spider.settings['CASE_ID_FILTER'] if spider.settings['CASE_ID_FILTER'] != None else True
            if case_id_filter:
                logger.info(f'>>>>{spider.name}:案例id去重开启<<<<')
                # todo 连接mongo集群需优化
                col_name_li: list = ['beike_xf', ]
                spider.filter_flag = get_filter_flag('newhours', col_name_li, 'beike_xf_pid',
                                                     mongo_client=MongodbClient()._client)
                logger.info(f'>>>>{spider.name}:案例id去重配置成功<<<<') if spider.filter_flag else logger.error(
                    f'>>>>{spider.name}:案例id去重配置失败<<<<')
            else:
                spider.filter_flag = None
                logger.info(f'>>>>{spider.name}:不根据案例id进行去重<<<<')

    def process_item(self, item, spider):
        if spider.name == 'beike':
            # str空值处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item_str(item_tup[1])
            # list类型处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item_list(item_tup[1])
            # avgPrice字段
            item['avgPrice'] = self.list_to_str(item['avgPrice'])
            # buildingArea字段
            item['buildingArea'] = self.list_to_str(item['buildingArea'])
            # developerName字段
            item['developerName'] = self.list_to_str(item['developerName'])
            # mainStructure字段
            item['mainStructure'] = self.list_to_str(item['mainStructure'])
            # projectAddress字段
            item['projectAddress'] = self.list_to_str(item['projectAddress'])
            item['siteName'] = '贝壳'  # 房源（中文名）
            item['fetchTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')  # 爬取时间
        return item

    @staticmethod
    def deal_item_str(item_val):
        """
        案例字段str空值处理
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert isinstance(item_val, str)
            item_val = item_val.strip() if item_val.strip() else None
        except:
            return item_val
        else:
            return item_val

    @staticmethod
    def deal_item_list(item_val):
        """
        案例字段list空值处理
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert isinstance(item_val, list)
            item_val_d = list()
            for i in item_val:
                assert isinstance(i, str)
                i = i.strip()
                item_val_d.append(i) if i else False
            item_val_d = item_val_d if item_val_d else None
        except:
            return item_val
        else:
            return item_val_d

    @staticmethod
    def list_to_str(item_val):
        """
        list类型转str类型
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert item_val and isinstance(item_val, list)
            item_val = ''.join(item_val)
        except:
            return item_val
        else:
            return item_val


class AnjukePipeline(object):
    """
    安居客新房
    """

    def open_spider(self, spider):
        if spider.name == 'anjuke':
            case_id_filter = spider.settings['CASE_ID_FILTER'] if spider.settings['CASE_ID_FILTER'] != None else True
            if case_id_filter:
                logger.info(f'>>>>{spider.name}:案例id去重开启<<<<')
                # todo 连接mongo集群需优化
                col_name_li: list = ['anjuke_xf', ]
                spider.filter_flag = get_filter_flag('newhours', col_name_li, 'anjuke_xf_pid',
                                                     mongo_client=MongodbClient()._client)
                logger.info(f'>>>>{spider.name}:案例id去重配置成功<<<<') if spider.filter_flag else logger.error(
                    f'>>>>{spider.name}:案例id去重配置失败<<<<')
            else:
                spider.filter_flag = None
                logger.info(f'>>>>{spider.name}:不根据案例id进行去重<<<<')

    def process_item(self, item, spider):
        if spider.name == 'anjuke':
            # str空值处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item_str(item_tup[1])
            # list类型处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item_list(item_tup[1])
            # locationArea字段
            item['locationArea'] = self.list_to_str(item['locationArea'])
            # propertyCompany字段
            item['propertyCompany'] = self.list_to_str(item['propertyCompany'])
            # preSalePermit字段
            item['preSalePermit'] = self.list_to_str(item['preSalePermit'])
            # preSaleBuilding字段
            item['preSaleBuilding'] = self.list_to_str(item['preSaleBuilding'])
            # certDate字段
            item['certDate'] = self.list_to_str(item['certDate'])
            item['districtName'] = self.get_district_name(item['locationArea'])  # 行政区/县
            item['siteName'] = '安居客'  # 房源（中文名）
            item['fetchTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')  # 爬取时间
        return item

    @staticmethod
    def deal_item_str(item_val):
        """
        案例字段str空值处理
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert isinstance(item_val, str)
            item_val = item_val.strip() if item_val.strip() else None
        except:
            return item_val
        else:
            return item_val

    @staticmethod
    def deal_item_list(item_val):
        try:
            """
        案例字段list空值处理
        :param item_val: 字段的value值
        :return:
        """
            assert isinstance(item_val, list)
            item_val_d = list()
            for i in item_val:
                assert isinstance(i, str)
                i = i.strip()
                item_val_d.append(i) if i else False
            item_val_d = item_val_d if item_val_d else None
        except:
            return item_val
        else:
            return item_val_d

    @staticmethod
    def list_to_str(item_val):
        """
        list类型转str类型
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert item_val and isinstance(item_val, list)
            item_val = ''.join(item_val)
        except:
            return item_val
        else:
            return item_val

    @staticmethod
    def get_district_name(location: str):
        """
        提取行政区/县
        :param location:
        :return:
        """
        try:
            assert location
            ret = location.split('-')[0]
            assert ret
        except:
            return
        else:
            return ret
